package org.springside.examples.showcase.wb.service;

import java.io.File;
import java.io.FileNotFoundException;
import java.io.FileOutputStream;
import java.io.IOException;
import java.io.OutputStream;
import java.sql.ResultSet;
import java.sql.SQLException;
import java.util.ArrayList;
import java.util.Calendar;
import java.util.Collections;
import java.util.Comparator;
import java.util.HashMap;
import java.util.Iterator;
import java.util.List;
import java.util.Map;

import org.apache.commons.io.FileUtils;
import org.apache.commons.io.IOUtils;
import org.apache.commons.lang.StringUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.beans.BeanUtils;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.jdbc.core.RowMapper;
import org.springframework.stereotype.Component;
import org.springframework.transaction.annotation.Transactional;
import org.springside.examples.showcase.jms.MyNotifyMessageProducer;
import org.springside.examples.showcase.utilities.wb.SinaCrawler;
import org.springside.examples.showcase.utilities.wb.SinaMid2Url;
import org.springside.examples.showcase.utilities.wb.SinaUnicode2GB;
import org.springside.examples.showcase.utilities.wb.SinaWbParse;
import org.springside.examples.showcase.utilities.wb.TokenUtils;
import org.springside.examples.showcase.wb.dao.CrawlerCommentsRecordsHibernateDao;
import org.springside.examples.showcase.wb.dao.TopKeyWordJpaDao;
import org.springside.examples.showcase.wb.dao.TopKeywordHibernateDao;
import org.springside.examples.showcase.wb.dao.WbContentHibernateDao;
import org.springside.examples.showcase.wb.dao.WbContentJdbcDao;
import org.springside.examples.showcase.wb.dao.WbContentJpaDao;
import org.springside.examples.showcase.wb.dao.WbContentMyBatisDao;
import org.springside.examples.showcase.wb.dao.WbKeywordJpaDao;
import org.springside.examples.showcase.wb.entity.CrawlerCommentsRecords;
import org.springside.examples.showcase.wb.entity.KvNode;
import org.springside.examples.showcase.wb.entity.TopKeyWord;
import org.springside.examples.showcase.wb.entity.WbContent;
import org.springside.examples.showcase.wb.entity.WbContentRet;
import org.springside.examples.showcase.wb.entity.WbKeywords;
import org.springside.modules.orm.Page;
import org.springside.modules.utils.RegexUtil;

/**
 * 用户管理类.
 * 
 * @author calvin
 */
// Spring Service Bean的标识.
@Component
@Transactional
public class WbContentManager {
	private static Logger logger = LoggerFactory.getLogger(WbContentManager.class);
	private WbContentJpaDao wbContentJpaDao;
	private WbContentMyBatisDao wbContentMyBatisDao;
	private WbContentHibernateDao wbContentHibernateDao;
	private WbContentJdbcDao wbContentJdbcDao;
	private TopKeyWordJpaDao topKeyWordJpaDao;
	private WbKeywordJpaDao wbKeywordJpaDao;
	private TopKeywordHibernateDao topKeywordHibernateDao;
	private MyNotifyMessageProducer myNotifyMessageProducer;
	private CrawlerCommentsRecordsHibernateDao crawlerCommentsRecordsHibernateDao;
	private String hqlByPardate = "from TopKeyWord tw where pardate = ?";
	private String hqlByUidMid = "from WbContent wb where wb.ir_mid = ? and wb.ir_uid = ?";

	/**
	 * 确立某天的热点消息（根据转发情况频度定）
	 * 
	 * @param ir_created_date
	 * @return
	 */
	public int insertHotRttRelations(String ir_created_date) {
		WbContent wbContent = new WbContent();
		wbContent.setIr_created_date(ir_created_date);
		return wbContentMyBatisDao.insertHotRttRelations(wbContent);
	}

	/**
	 * 获取热点转发消息的内容
	 * 
	 * @param page
	 * @param pardate 那一天的热点
	 * @return
	 */
	public Page<WbContentRet> getHotWbs(Page<WbContentRet> page, String pardate) {
		List<WbContentRet> hotRetWbs = wbContentJdbcDao.getHotRetWbs(page, pardate);
		page.setResult(hotRetWbs);
		return page;
	}

	public List<WbContent> getWbByDate(Page<WbContent> page, String datestr) {
		page = wbContentMyBatisDao.getByCreatedDate(page, datestr);
		List<WbContent> rowsList = page.getResult();
		return rowsList;
	}

	/**
	 * 确定哪些消息，在数据库库中不存在，即需要实时下载的消息
	 * 
	 * @param pardate
	 * @return 微博消息对应的mid，及uid
	 */
	public List<WbContent> getNeedDownLoadFeeds(String pardate) {
		List<WbContent> needDownloadFeeds = wbContentJdbcDao.getNeedDownloadFeeds(pardate);
		return needDownloadFeeds;
	}

	/**
	 * 实时下载微博消息
	 * 
	 * @param uid 用户id
	 * @param mid 微博mid
	 * @return 解析后微博的java bean
	 */
	public WbContent downloadHotFeeds(String uid, String mid) {
		File htmlBak = new File(System.getProperty("user.dir") + "/data/" + uid + "/" + mid + ".html");
		String wbContentHtml = null;
		if (!htmlBak.exists()) {
			try {
				htmlBak.getParentFile().mkdirs();
				htmlBak.createNewFile();
			} catch (IOException e) {
				e.printStackTrace();
			}
			wbContentHtml = SinaCrawler.getWbContents("/" + uid + "/" + SinaMid2Url.mid2url("" + mid), null);
			wbContentHtml = SinaUnicode2GB.decode(wbContentHtml);
			try {
				OutputStream output = new FileOutputStream(htmlBak);
				IOUtils.write(wbContentHtml, output, "UTF-8");
			} catch (FileNotFoundException e) {
				e.printStackTrace();
			} catch (IOException e) {
				e.printStackTrace();
			}
		} else {
			try {
				wbContentHtml = FileUtils.readFileToString(htmlBak, "UTF-8");
			} catch (IOException e) {
				e.printStackTrace();
			}
		}
		WbContent wbContent = SinaWbParse.parseContent(wbContentHtml);
		return wbContent;
	}

	public void saveHotFeeds(WbContent wbContent) {
		wbContentHibernateDao.save(wbContent);
	}

	/**
	 * 将微博数据批量入库
	 * 
	 * @param wbList
	 */
	public void save(List<WbContent> wbList) {
		wbContentJpaDao.save(wbList);
	}

	public int updateDocs(String mids, String created_date) {
		return wbContentMyBatisDao.updateDocs(mids, created_date);
	}

	/**
	 * 获取需要分词的微博
	 * 
	 * @param pardate
	 * @return
	 */
	public List<WbContentRet> getNeedSegmentationFeeds(String pardate) {
		return wbContentJdbcDao.selects(NEEDSEGMENTFEEDS, new RowMapper<WbContentRet>() {
			@Override
			public WbContentRet mapRow(ResultSet rs, int rowNum) throws SQLException {
				WbContentRet wb = new WbContentRet();
				wb.setIr_mid(rs.getLong("ret_mid"));
				wb.setIr_status_content(rs.getString("ir_status_content"));
				wb.setIr_uid(rs.getString("ir_uid"));
				wb.setCount(rs.getInt("count"));
				return wb;
			}
		}, pardate);
	}

	public Map<String, KvNode> statKeywordFreq(List<WbContentRet> needSegmentationFeeds) {
		Map<String, KvNode> tokenMidMap = new HashMap<String, KvNode>();
		Map<String, WbContentRet> id2WbMap = new HashMap<String, WbContentRet>();
		for (WbContentRet wb : needSegmentationFeeds) {
			id2WbMap.put("" + wb.getIr_mid(), wb);
			File segmentsFile = new File(System.getProperty("user.dir") + "/data/" + wb.getIr_uid() + "/" + wb.getIr_mid() + ".txt");
			String tokens = "";
			if (!segmentsFile.exists()) {
				try {
					segmentsFile.getParentFile().mkdirs();
					segmentsFile.createNewFile();
					tokens = TokenUtils.doSegment(wb.getIr_status_content());
					OutputStream output = new FileOutputStream(segmentsFile);
					IOUtils.write(tokens, output, "UTF-8");
				} catch (IOException e) {
					e.printStackTrace();
				}
			} else {
				try {
					tokens = FileUtils.readFileToString(segmentsFile, "UTF-8");
				} catch (IOException e) {
					e.printStackTrace();
				}
			}
			String[] _tokens = tokens.split("\\,");
			for (String token : _tokens) {
				token = token.trim();
				if (token.length() > 1) {
					KvNode kvNode = tokenMidMap.get(token);

					if (kvNode == null) {
						kvNode = new KvNode();
						kvNode.setKey(token);
						kvNode.setInfo(wb.getIr_mid() + "");
						kvNode.setValue(wb.getCount());
						tokenMidMap.put(token, kvNode);
					} else if (!kvNode.getInfo().contains(wb.getIr_mid() + "")) {
						kvNode.setInfo(kvNode.getInfo() + "," + wb.getIr_mid());
						kvNode.setValue(kvNode.getValue() + wb.getCount());
					} else {
						kvNode.setValue(kvNode.getValue() + wb.getCount());
					}
				}
			}

		}
		return tokenMidMap;
	}

	public List<WbContent> getHotRttWbContents() {
		List<WbContent> wbcontents = (List<WbContent>) wbContentMyBatisDao.getHotRttWbs();
		return wbcontents;
	}

	/**
	 * 保存top N的热点词
	 * 
	 * @param pardate
	 * @param sortList
	 */
	public List<TopKeyWord> saveKeywords(String pardate, List<KvNode> sortList) {
		List<KvNode> tmpList = new ArrayList<KvNode>();
		int count = sortList.size() > 200 ? 200 : sortList.size();
		tmpList = sortList.subList(0, count);
		List<TopKeyWord> topKeyWords = new ArrayList<TopKeyWord>();
		for (Iterator<KvNode> iterator = tmpList.iterator(); iterator.hasNext();) {
			KvNode kvNode = iterator.next();
			TopKeyWord topKeyWord = new TopKeyWord();
			topKeyWord.setKeyword(kvNode.getKey());
			topKeyWord.setKeywordrel(kvNode.getKeyAlias());
			topKeyWord.setWeightavg((kvNode.getValue() / (kvNode.getInfo().split("\\,").length)));
			topKeyWord.setWeight(kvNode.getValue());
			topKeyWord.setRef_mid(kvNode.getInfo());
			topKeyWord.setPardate(pardate);
			topKeyWords.add(topKeyWord);
			logger.debug(kvNode.getKey() + " : " + (kvNode.getValue() / (kvNode.getInfo().split("\\,").length)) + " : " + kvNode.getValue() + " : " + kvNode.getInfo());
		}
		topKeyWordJpaDao.save(topKeyWords);
		return topKeyWords;
	}

	public List<WbKeywords> saveWbKeywords(List<WbContentRet> needSegmentationFeeds, List<KvNode> sortList, String pardate) {
		List<WbKeywords> wbKeywords = new ArrayList<WbKeywords>();
		for (WbContentRet wbret : needSegmentationFeeds) {
			WbKeywords wbKeyword = new WbKeywords();
			wbKeywords.add(wbKeyword);
			wbKeyword.setRet_mid(wbret.getIr_mid());
			wbKeyword.setRet_uid(wbret.getIr_uid());
			List<KvNode> kvNodes = new ArrayList<KvNode>();
			for (KvNode kvNode : sortList) {
				if (kvNode.getInfo().contains("" + wbret.getIr_mid())) {
					kvNodes.add(kvNode);
				}
			}
			Collections.sort(kvNodes, new Comparator<KvNode>() {
				@Override
				public int compare(KvNode o1, KvNode o2) {
					int k1 = o1.getInfo().split("\\,").length;
					int k2 = o2.getInfo().split("\\,").length;
					return (o2.getValue() / k2 - o1.getValue() / k1);
				}
			});
			String keywords = "";
			for (KvNode kvNode : kvNodes) {
				keywords += kvNode.getKey() + ":" + (kvNode.getValue() / kvNode.getInfo().split("\\,").length) + ",";
			}
			wbKeyword.setKeywords(keywords);
			wbKeyword.setPardate(pardate);
		}

		for (Iterator iterator = wbKeywords.iterator(); iterator.hasNext();) {
			WbKeywords kv = (WbKeywords) iterator.next();
			if (StringUtils.isEmpty(kv.getKeywords())) {
				iterator.remove();
			}
		}
		wbKeywordJpaDao.save(wbKeywords);
		return wbKeywords;
	}

	public List<TopKeyWord> getTopKeyWords(String pardate) {
		List<TopKeyWord> result = topKeywordHibernateDao.find(hqlByPardate, pardate);
		return result;
	}

	public WbContent get(Long ir_mid, String ir_uid) {
		WbContent wb = wbContentHibernateDao.findUnique(hqlByUidMid, ir_mid, ir_uid);

		if (wb == null) {
			wb = new WbContent();
			WbContent downloadHotFeeds = null;
			try {
				List<WbContent> needInsertFeeds = new ArrayList<WbContent>();
				downloadHotFeeds = downloadHotFeeds("" + ir_uid, "" + ir_mid);
				if (downloadHotFeeds.getIr_mid() == 0l || StringUtils.isEmpty(downloadHotFeeds.getIr_created_date())) {
					System.out.println("获取微博失败...");
				} else {
					BeanUtils.copyProperties(downloadHotFeeds, wb);
					needInsertFeeds.add(wb);
				}
				save(needInsertFeeds);
			} catch (Exception e) {
			}
		}

		return wb;
	}

	/**
	 * 检索跟帖
	 * 
	 * @param retweeted_mid
	 * @param uid
	 * @return
	 */
	public List<WbContentRet> getComments(String retweeted_mid, String uid, int limitcount) {
		List<WbContentRet> commentsList = wbContentJdbcDao.selects(COMMENTFEEDS, new RowMapper<WbContentRet>() {
			@Override
			public WbContentRet mapRow(ResultSet rs, int rowNum) throws SQLException {
				WbContentRet wb = new WbContentRet();
				wb.setIr_mid(rs.getLong("ir_mid"));
				wb.setIr_uid(rs.getString("ir_uid"));
				wb.setIr_status_content(rs.getString("ir_status_content"));
				wb.setIr_screen_name(rs.getString("ir_screen_name"));
				wb.setIr_created_at(rs.getTime("ir_created_at"));
				return wb;
			}
		}, retweeted_mid, limitcount);

		return commentsList;
	}

	/**
	 * 下载并缓存评论也的json数据，抽取某页的评论信息
	 * 
	 * @param uid
	 * @param commentsRecords
	 * @return
	 */
	public List<WbContent> downCommentFeeds(String uid, CrawlerCommentsRecords commentsRecords) {
		File htmlBak = new File(System.getProperty("user.dir") + "/data/" + uid + "/" + commentsRecords.getMid() + "_" + commentsRecords.getCrawler_at().getTime() + "_" + commentsRecords.getPage() + ".json");
		String wbContentHtml = null;
		if (!htmlBak.exists()) {
			try {
				htmlBak.getParentFile().mkdirs();
				htmlBak.createNewFile();
			} catch (IOException e) {
				e.printStackTrace();
			}

			// /aj/comment/big?id=3488460335784709&max_id=3488851098160956&page=36&_t=0&__rnd=1347291572614
			String url = String.format("/aj/comment/big?id=%s&max_id=%s&page=%s&_t=0&__rnd=%s", commentsRecords.getMid(), commentsRecords.getMax_comments_mid(), commentsRecords.getPage(), commentsRecords.getCrawler_at().getTime());
			wbContentHtml = SinaCrawler.getWbContents(url, "");
			wbContentHtml = SinaUnicode2GB.decode(wbContentHtml);
			try {
				OutputStream output = new FileOutputStream(htmlBak);
				IOUtils.write(wbContentHtml, output, "UTF-8");
			} catch (FileNotFoundException e) {
				e.printStackTrace();
			} catch (IOException e) {
				e.printStackTrace();
			}
		} else {
			try {
				wbContentHtml = FileUtils.readFileToString(htmlBak, "UTF-8");
			} catch (IOException e) {
				e.printStackTrace();
			}
		}

		Long time = htmlBak.lastModified();
		Calendar calendar = Calendar.getInstance();
		calendar.setTimeInMillis(time);
		List<WbContent> commentsByJson = SinaWbParse.parseComments(wbContentHtml, calendar);
		return commentsByJson;
	}

	/**
	 * 提取第一页的评论信息
	 * 
	 * @param retweeted_mid
	 * @param uid
	 * @return
	 */
	public List<WbContent> parserComments(String retweeted_mid, String uid) {
		File file = new File(System.getProperty("user.dir") + "/data/" + uid + "/" + retweeted_mid + ".html");
		Long time = file.lastModified();
		Calendar calendar = Calendar.getInstance();
		calendar.setTimeInMillis(time);
		try {
			String readFileToString = FileUtils.readFileToString(file, "UTF-8");
			List<WbContent> comments = SinaWbParse.parseComments(readFileToString, calendar);
			return comments;
		} catch (IOException e) {
		}
		return null;
	}

	/**
	 * 提取翻页信息，建立采集起始点
	 * 
	 * @param retweeted_mid
	 * @param uid
	 * @return
	 */
	public List<CrawlerCommentsRecords> parserCommentPages(String retweeted_mid, String uid) {
		File file = new File(System.getProperty("user.dir") + "/data/" + uid + "/" + retweeted_mid + ".html");
		Long time = file.lastModified();
		Calendar calendar = Calendar.getInstance();
		calendar.setTimeInMillis(time);
		try {
			String readFileToString = FileUtils.readFileToString(file, "UTF-8");
			List<String> parseComments = SinaWbParse.parseCommPageUrls(readFileToString);
			if (parseComments.size() == 7) {
				// id=3488460335784709&max_id=3488660723032481&page=26
				String firstpage = parseComments.get(parseComments.size() - 1);

				String mid = RegexUtil.getMatchGroup(firstpage, "(id=(.*?)&max_id)", 2);
				String max_mid = RegexUtil.getMatchGroup(firstpage, "(&max_id=(.*?)&page=)", 2);
				Integer pageCount = Integer.valueOf(RegexUtil.getMatchGroup(firstpage, "(&page=(.*))", 2).trim());

				List<CrawlerCommentsRecords> commentsRecords = new ArrayList<CrawlerCommentsRecords>();
				for (int i = pageCount; i > 1; i--) {
					CrawlerCommentsRecords commentsRecord = new CrawlerCommentsRecords();
					commentsRecord.setPage(i);
					commentsRecord.setMid(Long.valueOf(mid));
					commentsRecord.setMax_comments_mid(Long.valueOf(max_mid));
					commentsRecord.setCrawler_at(calendar.getTime());
					commentsRecords.add(commentsRecord);
				}
				return commentsRecords;
			}
		} catch (IOException e) {
		}
		return null;

	}

	@Autowired
	public void setWbContentJpaDao(WbContentJpaDao wbContentJpaDao) {
		this.wbContentJpaDao = wbContentJpaDao;
	}

	@Autowired
	public void setWbContentMyBatisDao(WbContentMyBatisDao wbContentMyBatisDao) {
		this.wbContentMyBatisDao = wbContentMyBatisDao;
	}

	@Autowired
	public void setWbContentHibernateDao(WbContentHibernateDao wbContentHibernateDao) {
		this.wbContentHibernateDao = wbContentHibernateDao;
	}

	@Autowired
	public void setWbContentJdbcDao(WbContentJdbcDao wbContentJdbcDao) {
		this.wbContentJdbcDao = wbContentJdbcDao;
	}

	@Autowired
	public void setTopKeyWordJpaDao(TopKeyWordJpaDao topKeyWordJpaDao) {
		this.topKeyWordJpaDao = topKeyWordJpaDao;
	}

	@Autowired
	public void setWbKeywordJpaDao(WbKeywordJpaDao wbKeywordJpaDao) {
		this.wbKeywordJpaDao = wbKeywordJpaDao;
	}

	@Autowired
	public void setTopKeywordHibernateDao(TopKeywordHibernateDao topKeywordHibernateDao) {
		this.topKeywordHibernateDao = topKeywordHibernateDao;
	}

	@Autowired
	public void setMyNotifyMessageProducer(MyNotifyMessageProducer myNotifyMessageProducer) {
		this.myNotifyMessageProducer = myNotifyMessageProducer;
	}

	@Autowired
	public void setCrawlerCommentsRecordsHibernateDao(CrawlerCommentsRecordsHibernateDao crawlerCommentsRecordsHibernateDao) {
		this.crawlerCommentsRecordsHibernateDao = crawlerCommentsRecordsHibernateDao;
	}

	public WbContentJpaDao getWbContentJpaDao() {
		return wbContentJpaDao;
	}

	public WbContentMyBatisDao getWbContentMyBatisDao() {
		return wbContentMyBatisDao;
	}

	public WbContentHibernateDao getWbContentHibernateDao() {
		return wbContentHibernateDao;
	}

	public WbContentJdbcDao getWbContentJdbcDao() {
		return wbContentJdbcDao;
	}

	public static void main(String[] args) {
		new WbContentManager().downloadHotFeeds("2717762443", "yy9n7fCqM");
	}

	private static final String NEEDSEGMENTFEEDS = "select rt.ret_mid,rt.count,wb.ir_status_content,wb.ir_uid from ret_top rt inner join sina_wb wb on rt.ret_mid = wb.ir_mid left join sina_wb_keywords kw on rt.ret_mid = kw.ret_mid where rt.pardate=? and wb.ir_mid is not null and kw.ret_mid is null ";
	private static final String COMMENTFEEDS = "select * from  sina_wb wb where ir_retweeted_mid=? order by ir_created_at asc limit ?";

}
